#!/usr/bin/perl

#sleep 25000;

###########################
### Autor: Sebastian Enger / B.Sc. 
### Copyright: Sebastian Enger
### Licence: BSD
### Version: 1.1.b  - 20080716@19.Uhr
### Contact: sebastian.enger@gmail.com | icq: 135846444
### Latest Version: PENDRIVE:\Programming\Perl\IncrementalBackup
###########################

use strict;
use DBI;
use File::Find;	# perl -MCPAN -e 'force install "File::Find"'
use File::Copy;	# perl -MCPAN -e 'force install "File::Copy"'
use File::Path;	# perl -MCPAN -e 'force install "File::Path"'
#use XML::Code; # perl -MCPAN -e 'force install "Curses"'
use File::Basename;
use Data::Dumper;
use String::Random; # perl -MCPAN -e 'force install "String::Random"'
use Digest::SHA qw(sha512_hex);
use Encode qw(encode decode);
use HTML::LinkExtractor; # perl -MCPAN -e 'force install "HTML::LinkExtractor"'
# perl -MCPAN -e 'force install "URI"'
use HTML::Strip; # perl -MCPAN -e 'force install "HTML::Strip"'

my $scriptname 		= basename($0);
my $pass 			= new String::Random;
my $LX 				= new HTML::LinkExtractor(undef,undef,1);
my $strip			= HTML::Strip->new();

#my $SourceFolder 	= '/home/sumy/db/synonyme.woxikon.de/synonyme/gut.php';
my $SourceFolder 	= '/home/sumy/compressed/tmp/name/www.vorname.com/';
my $string;
our $rcount 		= 0;
our $count 			= 0;
my $Hash;
my $dbh;
my %Hash 			= ();

##system("mysqldump -u root -p###########!########### -f --skip-skip-lock-tables buzzerstar > buzzerstarv3.sql");exit;
my $enc 			= 'latin1'; # in dieser Kodierung ist das Script gespeichert

unlink "text/logfile_$scriptname.txt";
open(L,">text/logfile_$scriptname.txt");
binmode(L, ":$enc");

#eval {
find(\&SourceStructure, $SourceFolder);
#};
close L;

print "I found $count entries with overall $rcount Synonyms\n";
#$dbh->disconnect();
exit;

sub SourceStructure(){

    # $_ is set to the current file name
    # $File::Find::dir is set to the current directory
    # $File::Find::name is set to "$File::Find::dir/$_"
    # you are chdir()'d to $File::Find::dir 

	my $SourceFilename		= $File::Find::name;
	my $SourceDirname		= $File::Find::dir;
	my $string = "";
	
	if ($SourceFilename){	
		#next if (listen|lemma).php?
		#print "Reading: $SourceFilename\n";
		{
		  local $/ = undef;
		  open FILE, "$SourceFilename" or die "Couldn't open file: $SourceFilename $!";
		  binmode FILE;
		  $string = <FILE>;
		  close FILE;
		}
		
		my $dbh = DBI->connect("dbi:mysql:dbname=onetipp","root","###########99", 
			{
				RaiseError => 0, PrintError => 1, mysql_enable_utf8 => 1
			}
		);
				
				my $first;
				my (@first1) 			= ($string =~ m#<meta property="og:title"\s*(.*?)\s*/>#igs);
				my $first 				= join(" ",@first1);

				my $gender;
				if ($first=~/Jungennamen/ig){
					$gender = "M";
				} else {
					$gender = "F";
				}
				
				my $name;
				my (@name1) 			= ($string =~ m#<title>\s*(.*?)\s*</title>#igs);
				my $name_tmp			= join(" ",@name1);
				
				my ($name, undef)		= split(" Vorname - ",$name_tmp);
				chomp($name);
				next if $name 			=~ /(Vornamen|namen|Bedeutung)/ig;
				
				my $bedeutung;
				my (@name2) 			= ($string =~ m#<ul id="content_in">\s*(.*?)\s*</ul>#igs);
				my $name_tmp			= join(" ",@name2);
				my $bedeutung 			= $strip->parse( $name_tmp );
				$strip->eof;
				chomp($bedeutung);
				
				my $sprache;
				my (@name3) 			= ($string =~ m#<div class="titel_line">Sprachen\s*(.*?)\s*<div class="w311 fl_l ml14">#igs);
				my $name3_tmp			= join(" ",@name3);
				my $sprache 			= $strip->parse( $name3_tmp );
				$strip->eof;
				$sprache				= trim($sprache);
				#print $name3_tmp;
				#print "\n";
				#next;
				#print "$count) Name: $name [$gender] -> Sprache: $sprache -> Bedeutung: $bedeutung\n";
				
				#$first 					= &parse_single_link($tmp_first);
				#print Dumper $first; 
				#print "content_text_raw:$content_text_raw\n";
				
				#print Dumper $syn_db_ret; 
				
				next if $name		!~ /(\w{2,})/ig;
				next if $gender 	!~ /(\w{1,})/ig;
				next if $sprache 	!~ /(\w{2,})/ig;
				
				#print "$count ] first: $first und syn: $syn_db_ret\n";
				$count++;
											
				my $query = sprintf("%s (%s, %s, %s, %s, %s)",
					"INSERT INTO namen_table (name,gender,language,meaning,uid) VALUES",
					$dbh->quote($name),
					$dbh->quote($gender),	
					$dbh->quote($sprache),	
					$dbh->quote($bedeutung),	
					$dbh->quote(""),											
				);
				eval { $dbh->do($query) };
				$dbh->disconnect;
				
	}; # if ( $SourceFilename =~ /\.txt/ig ){#&& $SourceDirname =~ /Team/ig){
	
}; # sub SourceStructure(){

sub parse_single_link(){
	my $text_raw 	= shift;
	$LX->parse(\$text_raw);
	
	for my $Link( @{ $LX->links } ) {
		my $syn = $$Link{_TEXT};
		chomp($syn);
		return $syn;
	}
}
sub parse_synonyms(){
	my $text_raw 	= shift;
	my $first_word	= shift;
	
	#print "first_word: $first_word\n";
	#print "text_raw: $text_raw\n";
	
	my $syn_db		= "";
	
	chomp($first_word);
	return if length($first_word) < 3;
		
	#my ($language, $probability) = langof($first_word);
	#print "Language of ^$first_word^ is: $language with Wahrscheinlichkeit: $probability\n";
	#next if $language ne "de";
	
	$LX->parse(\$text_raw);
		
	my $tmp_count = 0;
	for my $Link( @{ $LX->links } ) {
		
		my $synonym = $$Link{_TEXT};
		chomp($synonym);
		
		next if $synonym 	eq $first_word; # keine doppelten einträge
		next if $synonym	=~ /Synonym eintragen/ig;
		$synonym 			=~ s/\[(\w+)\]//ig;	
		$synonym 			=~ s/\((\w+)\)//ig;	
		
		if ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
			$syn_db .= "$synonym;";
			$rcount++;
		}
	#	if($tmp_count == 0) {
	#	#	print "($language) -> First Word: $first_word -> Synonyms: $synonym; ";
	#		
	#	#	print L "First Word: $first_word -> Synonyms: $synonym; ";
	#		$rcount++;
	#		$tmp_count++;
	#	} elsif ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
	#	#	print "$synonym; ";
	#		
	#	#	print L "$synonym; ";
			
	#	}
	}
#	print L "\n";
#	print "\n\n";
	return $syn_db;
}

sub parse_synonyms_two(){
	my $text_raw 	= shift;
	my $syn_db		= "";
	
	my $first_word 	= "";
	
	$LX->parse(\$text_raw);
		
	my $tmp_count = 0;
	for my $Link( @{ $LX->links } ) {
		
		my $synonym = $$Link{_TEXT};
		chomp($synonym);
		
		if($tmp_count == 0) {
			$first_word = $synonym;
			$tmp_count++;
		}
		
		next if $synonym eq $first_word; # keine doppelten einträge
		next if $synonym =~ /standardsprachlich/igs;
		
		if ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
			$syn_db .= "$synonym;";
		}
		
	#	} elsif ($synonym=~/(\w{2,})/ig){# && $synonym=~/([a-zA-Z]{2,})/ig && length($synonym)>1){
	#	#	print "$synonym; ";
	#		
	#	#	print L "$synonym; ";
	#		$rcount++;
	#	}
	}
#	print L "\n";
#	print "\n\n";
	return ($first_word,$syn_db);
}

sub strip_tags(){
	my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	return $content;
}
sub strip_tags_array(){
	my @arr = shift;
	my @retArray = ();
	foreach my $content (@arr){
	#my $content = shift;
	$content =~ s|<.+?>||ig;
	$content =~ s|.+?>||ig;
	$content =~ s|content="||ig;
	$content =~ s|"||ig;
	push(@retArray, $content);
	#$content =~ s#<([^">]+(?:"[^"]+")*[^>]+)>##ig;
	}
	return @retArray;
}
sub links(){
	my $content = shift;
	my @links = $content =~ /href\s*=\s*"?([^"\s>]+)/gis;
	return @links;
}
sub ltrim { my $s = shift; $s =~ s/^\s+//;       return $s; };
sub rtrim { my $s = shift; $s =~ s/\s+$//;       return $s; };
sub  trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s; };